/*
 * Copyright (c) 2018-2024 北京华恒盛世科技有限公司
 * QUICKPOOL is licensed under Mulan PSL v2.
 * You can use this software according to the terms and conditions of the Mulan PSL v2.
 * You may obtain a copy of Mulan PSL v2 at:
 *     http://license.coscl.org.cn/MulanPSL2
 * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
 * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
 * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
 * See the Mulan PSL v2 for more details.
*/

#include "xdrbase.h"
#include "limit.h"
#include "jm.h"

char chosenPath[PATH_MAX];

static int set_jobcwd(struct jobEntry *, char *);
static bool_t set_jobfile_dir(char *, struct jobEntry *, struct lenData *);
static int unlink_job_stdfiles(char *, char *, struct jobEntry *);

int
rcp_file(struct jobInfo *jobbase, struct execution_file *xf, char *host, int op, char *errMsg)
{
    int cc, pid, i;
    STATUS_WAIT_T status;

    int p[2], eno;

    errMsg[0] = '\0';

    if (pipe(p) < 0) {
        snprintf(errMsg, MAXMESSAGELENGTH, "pipe(): %s", strerror(errno));
        return -1;
    }

    if ((pid = fork()) < 0) {
        snprintf(errMsg, MAXMESSAGELENGTH, "fork(): %s", strerror(errno));
        close(p[0]);
        close(p[1]);
        return -1;
    }

    if (pid == 0) {
        sigset_t newmask;
        int maxfds =  sysconf(_SC_OPEN_MAX);

        close(p[0]);

        dup2(p[1], 1);
        dup2(p[1], 2);

        i = open ("/dev/null", O_RDONLY, 0);
        dup2(i, 0);

        for (i = 1; i < NSIG; i++)
            Signal_(i, SIG_DFL);

        sigemptyset(&newmask);
        sigprocmask(SIG_SETMASK, &newmask, NULL);

        alarm(0);

        for (i = 3; i < maxfds; i++)
            close(i);

        if (getuid() == guidManager) {

            change_uid(guidManager);

            if (0 > setuid(jobbase->execute_uid)) {
                perror("setuid");
            }
        }

        char rcpCmd[PATH_MAX+1];
        if (baseParams[BASE_BINDIR].string_value != NULL) {
            snprintf(rcpCmd, PATH_MAX, "%s/qp file rcp", baseParams[BASE_BINDIR].string_value);
        } else {
            snprintf(rcpCmd, PATH_MAX, "qp file rcp");
        }

        char rcpArg[PATH_MAX+1];
        snprintf(rcpArg, PATH_MAX, "%s@%s:%s", jobbase->submit->user, host, xf->in_file);

        char strJobId[32];
        log_message(DEBUG, EXECUTE, "%s: Job %s rcpCmd <%s> infile <%s> outfile <%s> rcpArg <%s> options %x XF_OP_SUB2EXEC %x XF_OP_EXEC2SUB_APPEND %x op %x",
                    __func__, jobid_2longstring(jobbase->jobid, strJobId), rcpCmd,
                    xf->in_file, xf->out_file, rcpArg,
                    xf->direction, XF_OP_SUB2EXEC, XF_OP_EXEC2SUB_APPEND, op);

        // open_std_files send  XF_OP_SUB2EXEC send_results send XF_OP_EXEC2SUB
        char rcpsh[PATH_MAX+1];
        if (op & XF_OP_SUB2EXEC) {            
            if (xf->direction & XF_OP_SUB2EXEC_APPEND) {
                snprintf(rcpsh, PATH_MAX, "%s -a '%s' '%s'", rcpCmd, rcpArg, xf->out_file);
            } else {
                snprintf(rcpsh, PATH_MAX, "%s '%s' '%s'", rcpCmd, rcpArg, xf->out_file);
            }
            execlp("/bin/sh", "/bin/sh", "-c", rcpsh, NULL);
        } else {
            if (xf->direction & XF_OP_EXEC2SUB_APPEND) {
                snprintf(rcpsh, PATH_MAX, "%s -a '%s' '%s'", rcpCmd, xf->out_file, rcpArg);
            } else {
                snprintf(rcpsh, PATH_MAX, "%s '%s' '%s'", rcpCmd, xf->out_file, rcpArg);
            }
            execlp("/bin/sh", "/bin/sh", "-c", rcpsh, NULL);
        }

        perror(rcpCmd);
        exit(-1);
    }

    close(p[1]);

    cc = read(p[0], errMsg, 1024);
    for (i = cc; cc > 0;) {
        cc = read(p[0], errMsg+i, 1024);
        if (cc > 0)
            i += cc;
    }
    eno = errno;

    close(p[0]);

    if (i > 0) {
        errMsg[i] = '\0';

        if (errMsg[i-1] == '\n') {
            errMsg[i-1] = '\0';
        }
    } else {
        errMsg[0] = '\0';
    }

    if (cc < 0) {
        snprintf(errMsg, MAXMESSAGELENGTH, "read(): %s", strerror(eno));
        return -1;
    }

    if (waitpid(pid, &status, 0) < 0) {
        snprintf(errMsg, MAXMESSAGELENGTH, "waitpid(%d): %s", pid, strerror(errno));
        return -1;
    }

    if (JOB_WSTATUS(status)) {
        if (errMsg[0] == '\0') {
            snprintf(errMsg, MAXMESSAGELENGTH, "qp task rcp exited with non-zero status %x",
                     JOB_WSTATUS(status));
        }
        return -1;
    }

    return 0;
} // end function rcp_file

int
remove_jobfile(struct jobEntry *job)
{
    char strJobId[32];
    log_message(DEBUG2, EXECUTE, "%s: job <%s> jobfile <%s>",
                __func__, jobid_2longstring(job->job_base.jobid, strJobId),
                job->job_base.submit->script);

    if (!gf_isfullpath(job->job_base.submit->script)) {
        return -1;
    }

    char strJobFileDir[PATH_MAX+1];
    strncpy(strJobFileDir, job->job_base.submit->script, PATH_MAX);

    char *strJobFileName;
    strJobFileName = strrchr(strJobFileDir, '/');
    *strJobFileName = '\0';
    strJobFileName ++;

    return (unlink_job_stdfiles(strJobFileDir, strJobFileName, job));
} // end function remove_jobfile

int
setup_path_4job(struct jobEntry *job, struct lenData *jf)
{
    umask(job->job_base.submit->umask);

    char strJobId[32];
    log_message(DEBUG, EXECUTE, "%s: Job <%s> home_dir <%s> cwd <%s> execute_home <%s> execute_cwd <%s> job file script <%s> execute_uid %d.",
                __func__, jobid_2longstring(job->job_base.jobid, strJobId),
                job->job_base.submit->home_dir, job->job_base.submit->cwd,
                job->job_base.execute_home, job->job_base.execute_cwd,
                job->job_base.submit->script, job->job_base.execute_uid);

    char strFullPathFile[PATH_MAX+1];

    unsigned int i;
    for (i = 0; i < job->job_base.num_exec_files; i++) {

        if (gf_isfullpath(job->job_base.exec_files[i].in_file)) {
            continue;
        }

        if (gf_isfullpath(job->job_base.submit->cwd)) {
            snprintf(strFullPathFile, PATH_MAX, "%s/%s",
                     job->job_base.submit->cwd, job->job_base.exec_files[i].in_file);
        } else {
            snprintf(strFullPathFile, PATH_MAX, "%s/%s/%s",
                    job->job_base.submit->home_dir, job->job_base.submit->cwd,
                    job->job_base.exec_files[i].in_file);
        }

        strncpy(job->job_base.exec_files[i].in_file, strFullPathFile, PATH_MAX);
    }

    if (NULL == job->job_base.execute_cwd || '\0' == job->job_base.execute_cwd[0]) {

        char cwd[PATH_MAX+1];
        if (set_jobcwd(job, cwd) == -1) {
            log_message(DEBUG, EXECUTE, "%s: set_jobcwd(%s) failed", __func__, cwd);
            return -1;
        }

        FREEUP(job->job_base.execute_cwd);
        job->job_base.execute_cwd = copy_string(cwd);
        gf_setenv("PWD", cwd);
    } else {

        if (chdir(job->job_base.execute_cwd) < 0) {

            log_message(ERR, NOCLASS, "%s: Job <%s> chdir(%s) failed due to %m",
                        __func__, jobid_2longstring(job->job_base.jobid, strJobId),
                        job->job_base.execute_cwd);

            if (chdir(gstrTmpDir) < 0) {
                log_message(ERR, NOCLASS, "%s: Job <%s> chdir(%s) failed due to %m",
                            __func__, jobid_2longstring(job->job_base.jobid, strJobId),
                            gstrTmpDir);
            }

            FREEUP(job->job_base.execute_cwd);
            job->job_base.execute_cwd = copy_string(gstrTmpDir);
        }
        gf_setenv("PWD", job->job_base.execute_cwd);
    }

    char strJobFileName[PATH_MAX+1];
    if (NULL != job->job_base.execute_home && '\0' != job->job_base.execute_home[0]) {

        if (!gf_isfullpath(job->job_base.submit->script)) {

            snprintf(strFullPathFile, PATH_MAX, "%s/.quickpool/%s", job->job_base.execute_home,
                     job->job_base.submit->script);

            FREEUP(job->job_base.submit->script);
            job->job_base.submit->script = copy_string(strFullPathFile);

	    log_message(DEBUG, EXECUTE, "%s: job <%s> jobfile is %s.",
	                __func__, jobid_2longstring(job->job_base.jobid, strJobId),
                        job->job_base.submit->script);
        }

        gf_setenv("HOME", job->job_base.execute_home);
        gf_setenv("QP_JOBFILENAME", job->job_base.submit->script);


        if (0 >= strlen(job->job_base.submit->chkpnt_dir)) {
            snprintf(strJobFileName, PATH_MAX, "%s", job->job_base.submit->script);           
        } else {

            char *chkDirEnv;
            char chkDir[PATH_MAX+1];
            chkDirEnv = getenv("QP_CHKPNT_DIR");
            if (chkDirEnv != NULL ) {
                strncpy(chkDir, chkDirEnv, PATH_MAX);
                snprintf(strJobFileName, PATH_MAX, "%s/%s",
                         chkDir, strrchr(job->job_base.submit->script, '/') + 1);
            } else {
                snprintf(strJobFileName, PATH_MAX, "%s/%s",
                         job->job_base.submit->chkpnt_dir,
                         strrchr(job->job_base.submit->script, '/') + 1);
            }
        }

        gf_setenv("QP_CHKFILENAME", strJobFileName);
      
        return 0;
    }

    char strJobFileDir[PATH_MAX+1];
    if (!gf_isfullpath(job->job_base.submit->home_dir)
        || (NULL != job->job_base.submit->login_shell
            && '\0' != job->job_base.submit->login_shell[0])) {

        if (set_jobfile_dir(strJobFileDir, job, jf) < 0) {
            return -1;
        }
    } else {

        if (set_jobfile_dir(strJobFileDir, job, jf) < 0) {
            return -1;
        }
    }

    snprintf(strFullPathFile, PATH_MAX, "%s/%s", strJobFileDir, job->job_base.submit->script);
    FREEUP(job->job_base.submit->script);
    job->job_base.submit->script = copy_string(strFullPathFile);

    log_message(DEBUG, EXECUTE, "%s: job <%s>, job file dir <%s> with script name <%s>",
                __func__, jobid_2longstring(job->job_base.jobid, strJobId),
                strJobFileDir, job->job_base.submit->script);

    gf_setenv("QP_JOBFILENAME", job->job_base.submit->script);

    if (0 >= strlen(job->job_base.submit->chkpnt_dir)) {
        snprintf(strJobFileName, PATH_MAX, "%s", job->job_base.submit->script);
    } else {

        char *chkDirEnv;
        char chkDir[PATH_MAX+1];
        chkDirEnv = getenv("QP_CHKPNT_DIR");
        if (chkDirEnv != NULL ) {
            strncpy(chkDir, chkDirEnv, PATH_MAX);
            snprintf(strJobFileName, PATH_MAX, "%s/%s",
                     chkDir, strrchr(job->job_base.submit->script, '/') + 1);
        } else {
            snprintf(strJobFileName, PATH_MAX, "%s/%s", job->job_base.submit->chkpnt_dir,
                     strrchr(job->job_base.submit->script, '/') + 1);
        }
    }
    gf_setenv("QP_CHKFILENAME", strJobFileName);
  
    if (0 < strlen(strJobFileDir)) {

        char *sp;
        sp = strrchr(strJobFileDir, '/');

        if (sp != NULL && sp != strJobFileDir) {
            *sp = '\0';
        }

        FREEUP(job->job_base.execute_home);
        job->job_base.execute_home = copy_string(strJobFileDir);

        if (sp != NULL) {
            *sp = '/';
        }
    }

    log_message(DEBUG, EXECUTE, "%s: Job <%s> submit home <%s> cwd <%s> execute home <%s> execute cwd <%s> job script <%s> execute uid %d submit uid %d",
                __func__, jobid_2longstring(job->job_base.jobid, strJobId),
                job->job_base.submit->home_dir, job->job_base.submit->cwd,
                job->job_base.execute_home, job->job_base.execute_cwd,
                job->job_base.submit->script, job->job_base.execute_uid,
                job->job_base.submit->uid);

    return 0;
}

int
remove_dir_withfiles(char *dir, int recur)
{
#if defined(S_ISLNK) 
    char statBuf[MAXPATHLEN];

    if (readlink(dir, statBuf, MAXPATHLEN) != -1) { 
        return (unlink(dir));
    }
#endif

    DIR *dirp;
    if ((dirp = opendir(dir)) == NULL) {
        return -1;
    }

    struct dirent *dp;
    for (dp = readdir(dirp); dp != NULL; dp = readdir(dirp)) {

        if ((strcmp(dp->d_name, ".") == 0) || (strcmp(dp->d_name, "..") == 0) 
            || (strlen(dp->d_name) == 0)) { 

            continue;
        }

        char path[PATH_MAX+1];
        snprintf(path, PATH_MAX, "%s/%s", dir, dp->d_name);
        if (recur) {
            struct stat stBuf;
            int doRecur = 0;

            if (lstat(path , &stBuf) == 0) {
#if defined(S_ISLNK)
               if (!S_ISLNK(stBuf.st_mode)) {
                   doRecur = 1;
                }
#endif
            }

            if (doRecur) {
                remove_dir_withfiles(path, 1);
            }
	}

	rmdir(path);
        unlink(path);
    }

    closedir(dirp);
    return(rmdir(dir));
}

/* ------------------------------------------------
 *
 *  below are static functions used in this file
 *
 * ------------------------------------------------ */

static int
unlink_job_stdfiles(char *jobfiledir, char *jobfile, struct jobEntry *job)
{
    char strJobId[32];
    log_message(DEBUG2, EXECUTE, "%s: job <%s> job file dir <%s> with script name <%s>",
                __func__, jobid_2longstring(job->job_base.jobid, strJobId), jobfiledir, jobfile);

    char strJobFile[PATH_MAX+1];
    if (NULL != jobfiledir && 0 < strlen(jobfiledir)) {
        snprintf(strJobFile, PATH_MAX, "%s/%s", jobfiledir, jobfile);
    } else {
        snprintf(strJobFile, PATH_MAX, "%s", jobfile);
    }

    bool_t error = FALSE;

    char strUnlinkFile[PATH_MAX+1];
    snprintf(strUnlinkFile, PATH_MAX, "%s", strJobFile);

    int ret;
    ret = unlink(strUnlinkFile);
    if (0 > ret && errno != ENOENT) {

        log_message(ERR, NOCLASS, STR_JOB_FAIL_S_S_M, __func__,
                    jobid_2string(job->job_base.jobid, strJobId), "unlink", strUnlinkFile);
        error = TRUE;
    }

    if (!(job->job_base.job_attribute & JOB_SAVE_OUTPUT)) {

        snprintf(strUnlinkFile, PATH_MAX, "%s.out", strJobFile);
        ret = unlink(strUnlinkFile);
        if (0 > ret && errno != ENOENT) {

            log_message(ERR, NOCLASS, STR_JOB_FAIL_S_S_M, __func__,
                        jobid_2string(job->job_base.jobid, strJobId), "unlink", strUnlinkFile);
            error = TRUE;
        }
    }

    snprintf(strUnlinkFile, PATH_MAX, "%s.err", strJobFile);
    ret = unlink(strUnlinkFile);
    if (0 > ret && errno != ENOENT) {

        log_message(ERR, NOCLASS, STR_JOB_FAIL_S_S_M, __func__,
                    jobid_2string(job->job_base.jobid, strJobId), "unlink", strUnlinkFile);
        error = TRUE;
    }

    snprintf(strUnlinkFile, PATH_MAX, "%s.in", strJobFile);
    ret = unlink(strUnlinkFile);
    if (0 > ret && errno != ENOENT) {

        log_message(ERR, NOCLASS, STR_JOB_FAIL_S_S_M, __func__,
                    jobid_2string(job->job_base.jobid, strJobId), "unlink", strUnlinkFile);
        error = TRUE;
    }

    if (error) {
        return -1;
    }

    return 0;
} // end function unlink_job_stdfiles

static int
set_jobcwd(struct jobEntry *job, char *cwd)
{
    struct passwd *pw;
    if (gf_isfullpath(job->job_base.submit->cwd)) {

        strncpy(cwd, job->job_base.submit->cwd, PATH_MAX);
        if (chdir(cwd) == 0) {
            strncpy(cwd, chosenPath, PATH_MAX);
            return 0;
        }

        char strJobId[32];
        log_message(DEBUG, EXECUTE, "%s: chdir(%s) failed for job <%s>: %m.",
            __func__, chosenPath, jobid_2longstring(job->job_base.jobid, strJobId));

        pw = get_user_byname(job->job_base.execute_user);
        if (NULL != pw && NULL != pw->pw_dir && gf_isfullpath(pw->pw_dir)) {

            if (chdir(pw->pw_dir) == 0) {
                char homePath[PATH_MAX+1];
                char subCwd[PATH_MAX+1];
                int i;

                if (getcwd(homePath, sizeof(homePath))) {

                    strncpy(subCwd, job->job_base.submit->cwd, PATH_MAX);

                    for (i = 0; homePath[i] != '\0' && subCwd[i] == homePath[i]; i++);

                    if ((i != 0) && (homePath[i] == '\0')) {
                        if (subCwd[i] == '\0') {
                            strncpy(cwd, pw->pw_dir, PATH_MAX);
                        } else if (subCwd[i] == '/') {
                            strncpy(subCwd, subCwd+i+1, PATH_MAX);
                            snprintf(cwd, PATH_MAX, "%s/%s", pw->pw_dir, subCwd);
                        }

                        if (chdir(cwd) == 0)
                            return 0;
                    }
                }
            }
        }

        strncpy(cwd, gstrTmpDir, PATH_MAX);
        if (chdir(cwd) == -1) {

	    char strJobId[32];
	    log_message(DEBUG, EXECUTE, "%s: chdir(%s) failed after chdir(%s) failed for job <%s>: %m",
	                __func__, cwd, chosenPath, jobid_2longstring(job->job_base.jobid, strJobId));
            return -1;
        }

        return 0;
    }


    if (job->job_base.submit->cwd[0] == '\0') {
        strncpy(cwd, job->job_base.submit->home_dir, PATH_MAX);
    } else {
        snprintf(cwd, PATH_MAX, "%s/%s",
                 job->job_base.submit->home_dir, job->job_base.submit->cwd);
    }

    if (chdir(cwd) == 0) {
        strncpy(cwd, chosenPath, PATH_MAX);
        return 0;
    }

    pw = get_user_byname(job->job_base.execute_user);
    if (NULL != pw && NULL != pw->pw_dir && gf_isfullpath(pw->pw_dir)) {

        if (job->job_base.submit->cwd[0] == '\0') {
            strncpy(cwd, pw->pw_dir, PATH_MAX);
        } else {
            snprintf(cwd, PATH_MAX, "%s/%s", pw->pw_dir, job->job_base.submit->cwd);
        }

        if (chdir(cwd) == 0)
            return 0;

        char strJobId[32];
        log_message(DEBUG, EXECUTE, "%s: chdir(%s) failed for job <%s>: %m",
	            __func__, cwd, jobid_2longstring(job->job_base.jobid, strJobId));
    }


    strncpy(cwd, gstrTmpDir, PATH_MAX);
    if (chdir(cwd) == -1) {

        char strJobId[32];
        log_message(DEBUG, EXECUTE, "%s: chdir tmp (%s) failed for job <%s>: %m",
                    __func__, cwd, jobid_2longstring(job->job_base.jobid, strJobId));
        return -1;
    }

    return 0;
}

static int
create_jobfile(char *jobfiledir, char *chkpntdir,struct jobEntry *job, struct lenData *jf)
{
    struct jobInfo *jobBase = &job->job_base;

    char strJobFile[PATH_MAX+1];
    if (NULL == chkpntdir) {
        snprintf(strJobFile, PATH_MAX, "%s/%s", jobfiledir, jobBase->submit->script);
    } else {
        snprintf(strJobFile, PATH_MAX, "%s/%s", chkpntdir, jobBase->submit->script);
    }

    char strJobId[32];

    int fd;
    if ((fd = open(strJobFile, O_CREAT | O_TRUNC | O_WRONLY, 0700)) == -1) {
        log_message(ERR, NOCLASS, "%s: Job <%s> open jobfile[%s] failed: %m.",
                    __func__, jobid_2string(jobBase->jobid, strJobId), strJobFile);
        return -1;
    }

    char *shellLine;
    shellLine = (char *)SHELLLINE;
    unsigned int len = strlen(SHELLLINE);
    if (write(fd, shellLine, len) != len) {

        log_message(DEBUG, EXECUTE, "%s:Job <%s> write jobfile %s len=%lufailed: %m.",
                    __func__, jobid_2string(jobBase->jobid, strJobId), strJobFile,
                    strlen(shellLine) + 1);
        close(fd);
        return -1;
    }

    char *sp;
    if ((sp = strstr(jf->data, TRAPSIGCMD)) == NULL) {

        log_message(DEBUG, EXECUTE, "%s: No TRAPSIGCMD <$QP_TRAPSIGS...> is found in jobfile data for job <%s>",
                    __func__, jobid_2string(jobBase->jobid, strJobId));
        sp = jf->data;
    }

    len = strlen(sp);
    if (write(fd, sp, len) != len) {

        log_message(DEBUG, EXECUTE, "%s: Job <%s> write jobfile %s failed due to %m.",
                    __func__, jobid_2string(jobBase->jobid, strJobId), strJobFile);
        close(fd);
        return -1;
    }

    if (close(fd) == -1) {

        log_message(DEBUG, EXECUTE, "%s: Job <%s> close jobfile %s failed due to %m",
                    __func__, jobid_2string(jobBase->jobid, strJobId), strJobFile);
        return -1;
    }

    if (NULL != chkpntdir) {

        char strChkpntFileLink[PATH_MAX+1];
        snprintf(strChkpntFileLink, PATH_MAX, "%s/%s", jobfiledir, jobBase->submit->script);
        if ((symlink(strJobFile, strChkpntFileLink) == -1) && (errno != EEXIST)) {

            log_message(ERR, NOCLASS, "%s: unable to creat symbolic link from <%s> to <%s> for job <%s> due to %m.",
                        __func__, strChkpntFileLink, strJobFile,
                        jobid_2string(jobBase->jobid, strJobId));
            return -1;
        }
    }
    
    return 0;
}

static int
localize_job_restartfiles(char *jobfiledir, char *restartdir, struct jobInfo *jobbase)
{
    char strJobFile[PATH_MAX+1];
    snprintf(strJobFile, PATH_MAX, "%s", jobbase->submit->script);

    char strFullPathFile[PATH_MAX+1];
    if (NULL != jobfiledir && 0 < strlen(jobfiledir)) {
        snprintf(strFullPathFile, PATH_MAX, "%s/%s", jobfiledir, strJobFile);
    } else {
        snprintf(strFullPathFile, PATH_MAX, "%s", strJobFile);
    }

    char strJobId[32];

    char strFileLink[PATH_MAX+1];
    snprintf(strFileLink, PATH_MAX, "%s/%s", restartdir, strJobFile);

    if (symlink(strFileLink, strFullPathFile) == -1 && errno != EEXIST) {
        log_message(ERR, NOCLASS, "%s: unable to creat a symbolic link from <%s> to <%s> for job <%s> due to %m.",
                    __func__, strFileLink, strFullPathFile,
                   jobid_2string(jobbase->jobid, strJobId));
        return -1;
    }

    if (NULL != jobfiledir && 0 < strlen(jobfiledir)) {
        snprintf(strFullPathFile, PATH_MAX, "%s/%s.out", jobfiledir, strJobFile);
    } else {
        snprintf(strFullPathFile, PATH_MAX, "%s.out", strJobFile);
    }

    snprintf(strFileLink, PATH_MAX, "%s/%s.out", restartdir, strJobFile);

    if (symlink(strFileLink, strFullPathFile) == -1 && errno != EEXIST) {
        log_message(ERR, NOCLASS, "%s: unable to creat a symbolic link from <%s> to <%s> for job <%s> due to %m.",
                    __func__, strFileLink, strFullPathFile,
                   jobid_2string(jobbase->jobid, strJobId));
        return -1;
    }

    if (NULL != jobfiledir && 0 < strlen(jobfiledir)) {
        snprintf(strFullPathFile, PATH_MAX, "%s/%s.err", jobfiledir, strJobFile);
    } else {
        snprintf(strFullPathFile, PATH_MAX, "%s.err", strJobFile);
    }

    snprintf(strFileLink, PATH_MAX, "%s/%s.err", restartdir, strJobFile);

    if (symlink(strFileLink, strFullPathFile) == -1 && errno != EEXIST) {
        log_message(ERR, NOCLASS, "%s: unable to creat a symbolic link from <%s> to <%s> for job <%s> due to %m.",
                    __func__, strFileLink, strFullPathFile,
                   jobid_2string(jobbase->jobid, strJobId));
        return -1;
    }

    return 0;
}

static int
create_chkpnt_jobfile(char *jobfiledir, struct jobEntry *job, struct lenData *jf,
                      char *chkpntdir, char *restartdir)
{
    char *strDir;
    strDir = strrchr(job->job_base.submit->chkpnt_dir, '/');
    if (NULL == strDir || !(islong(strDir+1))) {

        log_message(ERR, NOCLASS, "%s: %s is not a valid check point directory.",
                    __func__, job->job_base.submit->chkpnt_dir);
        childjm_set_jobstatus(JOB_STATUS_PEND, REASON_JM_CHKPNT_DIR, job);
    }

    char strJobId[32];

    char strNewChkpntDir[PATH_MAX+1];
    char oldJobId[32], newJobId[32];
    if ('/' == job->job_base.submit->chkpnt_dir[0]) {

        snprintf(chkpntdir, PATH_MAX, "%s", job->job_base.submit->chkpnt_dir);

        *strDir = '\0';
        snprintf(strNewChkpntDir, PATH_MAX, "%s", job->job_base.submit->chkpnt_dir);

        snprintf(oldJobId, 32, "%s", strDir+1);
        snprintf(newJobId, 32, "%s", jobid_2string(job->job_base.jobid, strJobId));
        *strDir = '/';
    } else {

        if (NULL == job->job_base.execute_cwd || '\0' == job->job_base.execute_cwd[0]) {
            char cwd[PATH_MAX];

            if (set_jobcwd(job, cwd) == -1) {
                log_message(ERR, NOCLASS, "%s: cannot set execute cwd for job <%s>",
                            __func__, jobid_2string(job->job_base.jobid, strJobId));
                return -1;
            }

            FREEUP(job->job_base.execute_cwd);
            job->job_base.execute_cwd = copy_string(cwd);
        }

        snprintf(chkpntdir, PATH_MAX, "%s/%s",
                 job->job_base.execute_cwd, job->job_base.submit->chkpnt_dir);

        *strDir = '\0';
        snprintf(strNewChkpntDir, PATH_MAX, "%s/%s",
                 job->job_base.execute_cwd, job->job_base.submit->chkpnt_dir);

        snprintf(oldJobId, 32, "%s", strDir+1);
        snprintf(newJobId, 32, "%s", jobid_2string(job->job_base.jobid, strJobId));
        *strDir = '/';
    }

    if (!(job->job_base.submit->options & SUBOPT_RESTART)) {
        mode_t mode;
        mode = umask(0);
        if (job->job_base.submit->chkpnt_dir[0] == '/'
            && (strDir = strrchr(job->job_base.submit->chkpnt_dir, '/'))
                && strDir != &(job->job_base.submit->chkpnt_dir[0])) {

            *strDir = '/';
            if ((mkdir(strNewChkpntDir, 0777) < 0 && errno != EEXIST)
                || (mkdir(chkpntdir, 0700) < 0 && errno != EEXIST)) {

                *strDir = '/';
                log_message(ERR, NOCLASS, "%s: %s(%s) failed due to %m.", 
                            __func__, "mkdir", job->job_base.submit->chkpnt_dir);
                umask(mode);
                childjm_set_jobstatus(JOB_STATUS_PEND, REASON_JM_CHKPNT_DIR, job);
            }
        } else {
            if ((mkdir(strNewChkpntDir, 0777) < 0 && errno != EEXIST)
                || (mkdir(chkpntdir, 0700) < 0 && errno != EEXIST)) {

                log_message(ERR, NOCLASS, "%s: %s(%s) failed due to %m.",
                            __func__, "mkdir", job->job_base.submit->chkpnt_dir);
                umask(mode);
                childjm_set_jobstatus(JOB_STATUS_PEND, REASON_JM_CHKPNT_DIR, job);
            }
        }
        umask(mode);       

        if (create_jobfile(jobfiledir, chkpntdir, job, jf) < 0) {
            childjm_set_jobstatus(JOB_STATUS_PEND, REASON_JM_CREATE_JOBFILE, job);
            return -1;
        }

        return 0;
    }
    
    DIR *dirp1, *dirp2;
    snprintf(restartdir, PATH_MAX, "%s/%s", strNewChkpntDir, newJobId);

    if (strcmp(oldJobId, newJobId) != 0) {
        if (((dirp1 = opendir(chkpntdir)) == NULL) && (errno == ENOENT)) {
            if (((dirp2 = opendir(restartdir)) == NULL) && (errno == ENOENT)) {

                log_message(ERR, NOCLASS, "%s: %s(%s) failed due to: %m.",
                            __func__, "opendir", chkpntdir);
                return -1;
            } else {
                if (dirp2) {

                    (void) closedir(dirp2);
                } else {
                    log_message(ERR, NOCLASS, "%s: check point directory %s isn't accessable: %m.",
                                __func__, chkpntdir);
                }
            }
        } else {
            if (dirp1) {
                (void) closedir(dirp1);
            } else {

                log_message(ERR, NOCLASS, "%s: check point directory %s isn't accessable: %m.",
                            __func__, chkpntdir);
            }

            if (((dirp2 = opendir(restartdir)) == NULL) && (errno == ENOENT)) {

                if (rename(chkpntdir, restartdir) == -1)
                    return -1;

                if ((symlink(restartdir, chkpntdir) == -1) && (errno != EEXIST))
                    return -1;
            } else {

                if (dirp2) {
                    (void) closedir(dirp2);
                } else {

                    log_message(ERR, NOCLASS, "%s: check point directory %s isn't accessable: %m.",
                                __func__, chkpntdir);
                }
            }
        }
    }

    if (localize_job_restartfiles(jobfiledir, restartdir, &(job->job_base)) < 0) {

        unlink_job_stdfiles(jobfiledir, job->job_base.submit->script, job);
        childjm_set_jobstatus(JOB_STATUS_PEND, REASON_JM_CREATE_JOBFILE, job);

        return -1;
    }

    return 0;
}

#define STDOUT_DIRECT  1
#define STDERR_DIRECT  2
static void
isfilebuf_stdout(char *filebuf, struct jobInfo *jobbase, int flag)
{
    char *strFile, *ext = NULL;
    if (flag == STDOUT_DIRECT) {
        ext = (char *)"out";
        strFile = jobbase->submit->output_file;
    } else if (flag == STDERR_DIRECT) {
        ext = (char *)"err";
        strFile = jobbase->submit->error_file;
    } else {
        return;
    }

    char lastChar;
    lastChar = strFile[strlen(strFile) - 1];

    bool_t isDir = FALSE, hasLastSlash = FALSE;
    if (lastChar == '/' || lastChar == '\\' ) {
        isDir = TRUE;
        hasLastSlash = TRUE;
    }

    struct stat st;
    bool_t isDirExist = FALSE;
    if (!stat(strFile, &st) && S_ISDIR(st.st_mode)) {
        isDir = TRUE;
        isDirExist = TRUE;
    }

    if (isDir && !isDirExist) {

        if (mkdir(strFile, 0700) == 0) {
            isDirExist = TRUE;
        }
    }

    char strJobId[32];
    if (isDir && hasLastSlash) {
        snprintf(filebuf, PATH_MAX, "%s%s.%s",
                 strFile, jobid_2longstring(jobbase->jobid, strJobId), ext);
    } else if (isDir) {
        snprintf(filebuf, PATH_MAX, "%s/%s.%s",
                 strFile, jobid_2longstring(jobbase->jobid, strJobId), ext);
    } else {
        snprintf(filebuf, PATH_MAX, "%s", strFile);
    }

    return;
}

static int
symlink_stdout(char *jobfile, char *ext, struct jobInfo *jobbase)
{
    char fileLink[PATH_MAX+1];
    char fullpath[PATH_MAX+1];

    snprintf(fileLink, PATH_MAX, "%s.%s", jobfile, ext);

    fullpath[0] = '\0';
    if (chosenPath[0] != '/') {

        gf_getwd(fullpath);
        if (strlen(fullpath) && (fullpath[strlen(fullpath)-1] != '/')) {
            strncat(fullpath, "/", PATH_MAX);
        }
    }
    strncat(fullpath, chosenPath, PATH_MAX);

    if ((symlink(fullpath, fileLink) == -1) && (errno != EEXIST)) {
        char strJobId[32];
        log_message(ERR, NOCLASS, "%s:Job <%s> Unable to creat a symbolic link from <%s> to <%s>: %s",
                    __func__, jobid_2longstring(jobbase->jobid, strJobId), fileLink,
                    fullpath, strerror(errno));
        return -1;
    }

    return 0;
} // end function symlink_stdout

static int
open_std_files(char *jobfiledir, char *chkpntdir, struct jobEntry *job)
{
    unsigned int i;
    char xMsg[MAXMESSAGELENGTH+1], rcpMsg[MAXMESSAGELENGTH+1];
    struct jobInfo *jobBase = &(job->job_base);
    char errMsg[MAXMESSAGELENGTH+1];
    char filebuf[PATH_MAX+1], filebufLink[PATH_MAX+1];
    char jobFileLink[PATH_MAX+1];
    
    int jobfileDotOutIsLink = FALSE;
    int jobfileDotErrIsLink = FALSE;

    log_message(DEBUG, EXECUTE, "%s, the job file dir is <%s> and the job file is <%s>.",
                __func__, jobfiledir, jobBase->submit->script);

    char strJobFile[PATH_MAX+1];
    if (NULL == chkpntdir) {
        if (NULL != jobfiledir && 0 < strlen(jobfiledir)) {
            snprintf(strJobFile, PATH_MAX, "%s/%s", jobfiledir, jobBase->submit->script);
        } else {
            snprintf(strJobFile, PATH_MAX, "%s", jobBase->submit->script);
        }
    } else {
        snprintf(strJobFile, PATH_MAX, "%s/%s", chkpntdir, jobBase->submit->script);
        if (NULL == jobfiledir || 0 >= strlen(jobfiledir)) {
            snprintf(jobFileLink, PATH_MAX, "%s", jobBase->submit->script);
        } else {
            snprintf(jobFileLink, PATH_MAX, "%s/%s", jobfiledir, jobBase->submit->script);
        }
    }

    xMsg[0] = '\0';
    for (i = 0; i < jobBase->num_exec_files; i++) {
        if (jobBase->exec_files[i].direction & XF_OP_SUB2EXEC) {

            if (rcp_file(jobBase, jobBase->exec_files+i, jobBase->submit->from_node,
                         XF_OP_SUB2EXEC, rcpMsg) < 0) {

                snprintf(xMsg, MAXMESSAGELENGTH, "%s: Failed to copy file <%s> from submission host <%s> to file <%s> on execution host: %s\n",
                         __func__, jobBase->exec_files[i].in_file, jobBase->submit->from_node,
                         jobBase->exec_files[i].out_file, rcpMsg);
            } else {
                if (rcpMsg[0] != '\0')
                    snprintf(xMsg, MAXMESSAGELENGTH, "%s: Copy file <%s> from submission host <%s> to file <%s> on execution host: %s\n",
                             __func__, jobBase->exec_files[i].in_file, jobBase->submit->from_node,
                             jobBase->exec_files[i].out_file, rcpMsg);
            }
        }
    }

    if ((NULL != jobBase->submit->input_file && '\0' != jobBase->submit->input_file[0])) {
        strncpy(filebuf, jobBase->submit->input_file, PATH_MAX);
    } else {
        strncpy(filebuf, NULLFILE, PATH_MAX);
    }

    char strJobId[32];

    i = open(filebuf, O_RDONLY, 0);
    if (i < 0) {
        struct execution_file xf;

        if (errno != ENOENT) {
            snprintf(xMsg, MAXMESSAGELENGTH, STR_FUNC_S_FAIL_M, __func__, "open", filebuf);
        } else {

            if (!gf_isfullpath(jobBase->submit->input_file)) {
                if (gf_isfullpath(jobBase->submit->cwd)) {
                    snprintf(xf.in_file, PATH_MAX, "%s/%s",
                             jobBase->submit->cwd, jobBase->submit->input_file);
                } else {
                    snprintf(xf.in_file, PATH_MAX, "%s/%s/%s",
                             jobBase->submit->home_dir, jobBase->submit->cwd,
                             jobBase->submit->input_file);
                }
            } else {
                strncpy(xf.in_file, jobBase->submit->input_file, PATH_MAX);
            }

            snprintf(xf.out_file, PATH_MAX, "%s.in", strJobFile);

            xf.direction = XF_OP_SUB2EXEC;
            strncpy(filebuf, xf.out_file, PATH_MAX);

            int ret;
            ret = rcp_file(jobBase, &xf, jobBase->submit->from_node, XF_OP_SUB2EXEC, rcpMsg);
            if (0 == ret) {

                i = open(filebuf, O_RDONLY, 0);
                if (0 > i) {
                    log_message(ERR, NOCLASS, "%s: Job <%s> open (%s) failed due to %m",
                                __func__, jobid_2longstring(jobBase->jobid, strJobId), filebuf);
                    return -1;
                }

                if (dup2(i, 0) == -1) {
                    log_message(ERR, NOCLASS, "%s: Job <%s> dup2(%d,0) stdin <%s>: %m",
                                __func__, jobid_2longstring(jobBase->jobid, strJobId),
                                i, jobBase->submit->input_file);
                    return -1;
                }
            } else {

                snprintf(xMsg, MAXMESSAGELENGTH, "%s: Unable to copy stdin file <%s> on submission host <%s> to <%s> on execution host: %s\n",
                         __func__, xf.in_file, jobBase->submit->from_node, xf.out_file, rcpMsg);
            }
        }
    } else {

        if (dup2(i, 0) == -1) {
            log_message(ERR, NOCLASS, "%s: Job <%s> dup2(%d,0) stdin <%s>: %m",
                        __func__, jobid_2longstring(jobBase->jobid, strJobId),
                        i, jobBase->submit->input_file);
            return -1;
        }
    }

    int outFlag = 0, errFlag = 0;
    if (NULL != jobBase->submit->output_file
        && 0 == strcmp(jobBase->submit->output_file, "/dev/null")) {

        snprintf(filebuf, PATH_MAX, "%s", jobBase->submit->output_file);
    } else {
        outFlag = 1;

        if ((NULL != jobBase->submit->output_file && '\0' != jobBase->submit->output_file[0])
            && gboolStdoutDirect) {

            isfilebuf_stdout(filebuf, jobBase, STDOUT_DIRECT);
        } else {
            snprintf(filebuf, PATH_MAX, "%s.out", strJobFile);
        }

        if (NULL != chkpntdir) {
            snprintf(filebufLink, PATH_MAX, "%s.out", jobFileLink);
        }      
    }

    log_message(DEBUG, EXECUTE, "%s: job <%s> filebuf/stdout file <%s> STDOUT_DIRECT=%s",
                __func__, jobid_2longstring(jobBase->jobid, strJobId),
                filebuf, gboolStdoutDirect?"True":"False");

    if (gboolStdoutDirect) {
        i = open(filebuf, O_WRONLY| O_CREAT| O_APPEND, 0666);
    } else {
        i = open(filebuf, O_WRONLY| O_CREAT| O_APPEND, 0600);
    }

    if (i < 0 && outFlag && gboolStdoutDirect) {

        snprintf(filebuf, PATH_MAX, "%s.out", strJobFile);
        log_message(DEBUG, EXECUTE, "%s: job <%s> filebuf/stdout file <%s>",
                    __func__, jobid_2longstring(jobBase->jobid, strJobId), filebuf);

        i = open(filebuf, O_WRONLY| O_CREAT| O_APPEND, 0600);
    } else {

        if (symlink_stdout(strJobFile, (char *)"out", jobBase) < 0) {
            return -1;
        }
        jobfileDotOutIsLink = TRUE;
    }

    if ((outFlag) && !(chkpntdir == NULL)) {

        if( jobfileDotOutIsLink ) {
            snprintf(filebuf, PATH_MAX, "%s.out", strJobFile);
        }

        if ((symlink(filebuf, filebufLink) == -1) && (errno != EEXIST)) {
            snprintf(errMsg, MAXMESSAGELENGTH, "%s: Job <%s> Unable to creat a symbolic link from <%s> to <%s>: %s", 
                     __func__, jobid_2string(jobBase->jobid, strJobId), filebufLink,
                     filebuf, strerror(errno));
            log_message(ERR, NOCLASS, errMsg);
            return -1;
        }
    }

    if (i < 0) {
        log_message(ERR, NOCLASS, STR_JOB_FAIL_S_S_M, __func__,
                    jobid_2string(jobBase->jobid, strJobId), "open", filebuf);
        return -1;
    }
    if (dup2(i, 1) == -1) {

        log_message(ERR, NOCLASS, "%s: Job <%s> dup(%d,1) stdout <%s> failed: %m",
                    __func__, jobid_2longstring(jobBase->jobid, strJobId), i, filebuf);
        return -1;
    }

    if (!(NULL != jobBase->submit->error_file && '\0' != jobBase->submit->error_file[0])
        || ((NULL != jobBase->submit->output_file && '\0' != jobBase->submit->output_file[0]
             && NULL != jobBase->submit->error_file && '\0' != jobBase->submit->error_file[0])
             && 0 == strcmp(jobBase->submit->error_file, jobBase->submit->output_file))) {

        if (dup2(1, 2) == -1 ){

	    log_message(ERR, NOCLASS, "%s: Job <%s> dup(1,2) stderr failed: %m",
                        __func__, jobid_2longstring(jobBase->jobid, strJobId));
            return -1;
        }
    } else {

        if (0 == strcmp(jobBase->submit->error_file, "/dev/null")) {
            snprintf(filebuf, PATH_MAX, "%s", jobBase->submit->error_file);
        } else {
            errFlag = 1;

            if (gboolStdoutDirect) {

                isfilebuf_stdout(filebuf, jobBase, STDERR_DIRECT);
            } else {
                snprintf(filebuf, PATH_MAX, "%s.err", strJobFile);
            }            

            if (chkpntdir != NULL) {
                snprintf(filebufLink, PATH_MAX, "%s.err", jobFileLink);
            }            
        }

        log_message(DEBUG, EXECUTE, "%s: job <%s> filebuf/stderr file <%s> STDOUT_DIRECT=%s",
                    __func__, jobid_2longstring(jobBase->jobid, strJobId),
                    filebuf, gboolStdoutDirect?"True":"False");

        if (gboolStdoutDirect) {
            i = open(filebuf, O_WRONLY | O_CREAT | O_APPEND, 0666);
        } else {
            i = open(filebuf, O_WRONLY | O_CREAT | O_APPEND, 0600);
        }

        if (i < 0 && errFlag && gboolStdoutDirect) {

            snprintf(filebuf, PATH_MAX, "%s.err", strJobFile);
            log_message(DEBUG, EXECUTE, "%s: job <%s> filebuf/stderr file <%s>",
                        __func__, jobid_2longstring(jobBase->jobid, strJobId), filebuf);

            i = open(filebuf, O_WRONLY| O_CREAT| O_APPEND, 0600);
        } else {

            if (symlink_stdout(strJobFile, (char *)"err", jobBase) < 0) {
                return -1;
            }
            jobfileDotErrIsLink = TRUE;
        }
        
        if ((errFlag) && !(chkpntdir == NULL)) {

            if (jobfileDotErrIsLink) {
                snprintf(filebuf, PATH_MAX, "%s.err", strJobFile);
            }

            if ((symlink(filebuf, filebufLink) == -1) && (errno != EEXIST)) {
                snprintf(errMsg, MAXMESSAGELENGTH, "%s:Job <%s> Unable to creat a symbolic link from <%s> to <%s>: %s", 
                         __func__, jobid_2string(jobBase->jobid, strJobId),
                         filebufLink, filebuf, strerror(errno));
                log_message(ERR, NOCLASS, errMsg);
                return -1;
            }
        }

        if (i < 0) {
	    log_message(ERR, NOCLASS, "%s: Job <%s> open(%s) stderr failed: %m",
	                __func__, jobid_2longstring(jobBase->jobid, strJobId), filebuf);
            return -1;
        }

        if (dup2(i, 2) == -1) {
	    log_message(ERR, NOCLASS, "%s: Job <%s> dup(%d,2) stderr <%s> failed: %m",
	                __func__, jobid_2longstring(jobBase->jobid, strJobId), i, filebuf);
            return -1;
        }
    }

    if (xMsg[0] != '\0') {
        fprintf(stderr, "%s", xMsg);
    }

    return 0;
} // end function open_std_files

static int
check_jobfile(char *jobfiledir, struct jobEntry *job, struct lenData *jf)
{
    struct stat st;
    int ret;
    ret = stat(jobfiledir, &st);
    if (0 > ret) {
        mkdir(jobfiledir, 0700);
    }

    char strJobId[32];
    if (NULL != jf) {

        if (NULL == job->job_base.submit->chkpnt_dir
            || '\0' == job->job_base.submit->chkpnt_dir[0]) {

            if (create_jobfile(jobfiledir, NULL, job, jf) < 0) {

                childjm_set_jobstatus(JOB_STATUS_PEND, REASON_JM_CREATE_JOBFILE, job);

                log_message(DEBUG, EXECUTE, "%s: create_jobfile() fail for job <%s>.",
                            __func__, jobid_2string(job->job_base.jobid, strJobId));

                return -1;
            }

            if (open_std_files(jobfiledir, NULL, job) < 0) {

                unlink_job_stdfiles(jobfiledir, job->job_base.submit->script, job);
                childjm_set_jobstatus(JOB_STATUS_PEND, REASON_JM_CREATE_JOBFILE, job);

                return -1;
            }
        } else {

            char strRestartDir[PATH_MAX+1], strChkpntDir[PATH_MAX+1];
            if (create_chkpnt_jobfile(jobfiledir, job, jf, strChkpntDir, strRestartDir) < 0) {

                childjm_set_jobstatus(JOB_STATUS_PEND, REASON_JM_CREATE_JOBFILE, job);

                log_message(DEBUG, NOCLASS, "%s: create_chkpnt_jobfile() fail for job <%s>.",
                            __func__, jobid_2string(job->job_base.jobid, strJobId));

                return -1;
            }

            if (open_std_files(jobfiledir, strChkpntDir, job) < 0) {

                unlink_job_stdfiles(strChkpntDir, job->job_base.submit->script, job);
                childjm_set_jobstatus(JOB_STATUS_PEND, REASON_JM_CREATE_JOBFILE, job);

                return -1;
            }
        }        
    }

    char strJobFile[PATH_MAX+1];
    if ((NULL != jobfiledir) && (0 < strlen(jobfiledir))) {
        snprintf(strJobFile, PATH_MAX, "%s/%s", jobfiledir, job->job_base.submit->script);
    } else {
        snprintf(strJobFile, PATH_MAX, "%s", job->job_base.submit->script);
    }

    if (stat(strJobFile, &st) == -1) {
        if (errno != ENOENT) {
            log_message(ERR, NOCLASS, STR_JOB_FAIL_S_S_M, __func__,
                        jobid_2string(job->job_base.jobid, strJobId), "stat", strJobFile);
        }
        return -1;
    }

    return 0;
} // end function check_jobfile

static int
set_jobfile_dir(char *jobfiledir, struct jobEntry *job, struct lenData *jf)
{
    struct passwd *pw;
    pw = get_user_byname(job->job_base.execute_user);
    if ((pw && pw->pw_dir && gf_isfullpath(pw->pw_dir))) {

        snprintf(jobfiledir, PATH_MAX, "%s/.quickpool", pw->pw_dir);

        if (check_jobfile(jobfiledir, job, jf) == 0) {
            gf_setenv("HOME", pw->pw_dir);
            return 0;
        }
    }

    char strJobTmpDir[PATH_MAX+1];
    get_tmpdir_4job(strJobTmpDir, job->job_base.jobid, job->job_base.execute_uid);
    if (0 == strlen(strJobTmpDir)) {
        return -1;
    }

    snprintf(jobfiledir, PATH_MAX, "%s", strJobTmpDir);

    if (pw && pw->pw_dir) {
        gf_setenv("HOME", pw->pw_dir);
    } else {
        gf_setenv("HOME", jobfiledir);
    }

    return (check_jobfile(jobfiledir, job, jf));
} // end function set_jobfile_dir

